from gevent import monkey
monkey.patch_all()
from gevent.pool import Pool
import importlib  # 动态导入模块
import schedule
import time

from settings import PROXIES_SPIDERS, RUN_SPIDER_INTERVAL
from core.proxy_validate.httpbin_validator import check_proxy
from core.db.mongo_pool import MongoPool
from utils.log import logger


class RunSpider(object):
    def __init__(self):
        # 创建Mongopool对象
        self.mongopool = MongoPool()
        self.pool = Pool()

    def get_spider_from_settings(self):
        for full_class_name in PROXIES_SPIDERS:
            moudle_name, class_name = full_class_name.rsplit('.', maxsplit=1)
            # 根据模块名，导入模块
            moudle = importlib.import_module(moudle_name)
            # 根据类名，从模块中，获取类
            cls = getattr(moudle, class_name)
            # 有类名，可以创建爬虫对象
            spider = cls()
            yield spider

    def run(self):
        # 根据配置文件信息，获取爬虫对象列表
        spiders = self.get_spider_from_settings()
        # 遍历爬虫对象列表
        for spider in spiders:
            # self.__execute_one_spider_task(spider)
            # 使用异步任务执行这个方法
            self.pool.apply_async(self.__execute_one_spider_task, args=(spider, ))
        self.pool.join()

    def __execute_one_spider_task(self, spider):
        # 　把处理一个代理爬虫的代码抽取到一个方法，用于处理爬虫任务
        try:
            # 遍历爬虫对象的get_proxies方法，获取代理ip
            for proxy in spider.get_proxy():
                # 检测代理ip
                new_proxy = check_proxy(proxy)
                # 如果speed不为-1，说明可用，写入数据库
                if new_proxy.speed != -1 and new_proxy.speed <= 2:
                    # 写入数据库
                    self.mongopool.insert_or_update(new_proxy)
        except Exception as e:
            logger.exception(e)
            logger.exception("爬虫{} 出现错误".format(spider))

    @classmethod
    def start(cls):
        # 创建本类对象
        rs = RunSpider()
        rs.run()

        # 每隔 RUN_SPIDER_INTERVAL 小时检查代理是否可用
        schedule.every(RUN_SPIDER_INTERVAL).hours.do(rs.run)
        while True:
            schedule.run_pending()
            time.sleep(5)


if __name__ == "__main__":
    RunSpider().run()

    # RunSpider.start()
